{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/train-images-idx3-ubyte.gz\n",
      "Extracting data/train-labels-idx1-ubyte.gz\n",
      "Extracting data/t10k-images-idx3-ubyte.gz\n",
      "Extracting data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import math\n",
    "from tensorflow.contrib.learn.python.learn.datasets.mnist import read_data_sets\n",
    "tf.set_random_seed(0)\n",
    "mnist = read_data_sets(\"data\", one_hot=True, reshape=False, validation_size=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start\n",
      "0.9892\n"
     ]
    }
   ],
   "source": [
    "def nn_3_0():\n",
    "     #remove logs\n",
    "    logdir = 'logs_3_0/'\n",
    "    maxstep = 6000+1\n",
    "    #learning_rate = 0.0003\n",
    "    \n",
    "    if tf.gfile.Exists(logdir):\n",
    "        tf.gfile.DeleteRecursively(logdir)\n",
    "        tf.gfile.MakeDirs(logdir)\n",
    "        \n",
    "    sess = tf.InteractiveSession()\n",
    "    \n",
    "    X = tf.placeholder(tf.float32, [None, 28, 28, 1])\n",
    "    Y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "    #K = 6  # first convolutional layer output depth \n",
    "    #L = 12  # second convolutional layer output depth\n",
    "    #M = 24  # third convolutional layer\n",
    "    K = 4  # first convolutional layer output depth \n",
    "    L = 8  # second convolutional layer output depth\n",
    "    M = 12  # third convolutional layer\n",
    "    N = 200  # fully connected layer\n",
    "    W1 = tf.Variable(tf.truncated_normal([5, 5, 1, K], stddev=0.1))  # 5x5 patch, 1 input channel, K output channels\n",
    "    B1 = tf.Variable(tf.ones([K])/10)\n",
    "    W2 = tf.Variable(tf.truncated_normal([5, 5, K, L], stddev=0.1))\n",
    "    B2 = tf.Variable(tf.ones([L])/10)\n",
    "    W3 = tf.Variable(tf.truncated_normal([4, 4, L, M], stddev=0.1))\n",
    "    B3 = tf.Variable(tf.ones([M])/10)\n",
    "    \n",
    "    W4 = tf.Variable(tf.truncated_normal([7 * 7 * M, N], stddev=0.1))\n",
    "    B4 = tf.Variable(tf.ones([N])/10)\n",
    "    W5 = tf.Variable(tf.truncated_normal([N, 10], stddev=0.1))\n",
    "    B5 = tf.Variable(tf.ones([10])/10)\n",
    "    \n",
    "    lr = tf.placeholder(tf.float32)\n",
    "    pkeep = tf.placeholder(tf.float32)\n",
    "    \n",
    "    stride = 1  # output is 28x28\n",
    "    Y1 = tf.nn.relu(tf.nn.conv2d(X, W1, strides=[1, stride, stride, 1], padding='SAME') + B1)\n",
    "    stride = 2  # output is 14x14\n",
    "    Y2 = tf.nn.relu(tf.nn.conv2d(Y1, W2, strides=[1, stride, stride, 1], padding='SAME') + B2)\n",
    "    stride = 2  # output is 7x7\n",
    "    Y3 = tf.nn.relu(tf.nn.conv2d(Y2, W3, strides=[1, stride, stride, 1], padding='SAME') + B3)\n",
    "    \n",
    "    # reshape the output from the third convolution for the fully connected layer\n",
    "    YY = tf.reshape(Y3, shape=[-1, 7 * 7 * M])\n",
    "    Y4 = tf.nn.relu(tf.matmul(YY, W4) + B4)\n",
    "    Ylogits = tf.matmul(Y4, W5) + B5\n",
    "    Y = tf.nn.softmax(Ylogits)\n",
    "    cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=Ylogits, labels=Y_)\n",
    "    \n",
    "    cross_entropy1 = tf.reduce_mean(cross_entropy)*100\n",
    "    correct_prediction = tf.equal(tf.argmax(Y_,1), tf.argmax(Y, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    \n",
    "       #summary\n",
    "    summary_cross_entropy =  tf.summary.scalar(\"cross_entropy\", cross_entropy1)\n",
    "    summary_accuracy = tf.summary.scalar('accuracy', accuracy)\n",
    "    merged_summary_op = tf.summary.merge([summary_cross_entropy, summary_accuracy])\n",
    "    train_writer = tf.summary.FileWriter(logdir+'train/', sess.graph)\n",
    "    test_writer = tf.summary.FileWriter(logdir+ 'test/')\n",
    "\n",
    "    # training, learning rate = 0.005\n",
    "    train_step = tf.train.AdamOptimizer(lr).minimize(cross_entropy)\n",
    "    tf.global_variables_initializer().run()\n",
    "    print 'start'\n",
    "    for i in range(maxstep):\n",
    "         # learning rate decay\n",
    "        max_learning_rate = 0.003\n",
    "        min_learning_rate = 0.0001\n",
    "        decay_speed = 2000.0 # 0.003-0.0001-2000=>0.9826 done in 5000 iterations\n",
    "        learning_rate = min_learning_rate + (max_learning_rate - min_learning_rate) * math.exp(-i/decay_speed)\n",
    "        \n",
    "        batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "        sess.run(train_step, feed_dict={X: batch_xs, Y_: batch_ys, lr:learning_rate, pkeep:0.75})\n",
    "        \n",
    "        if i %100 == 0:\n",
    "                summary, acc= sess.run([merged_summary_op, accuracy], feed_dict ={X: mnist.test.images, Y_: mnist.test.labels, pkeep:1.0})\n",
    "                test_writer.add_summary(summary, i)\n",
    "                \n",
    "        if i % 100 == 0:\n",
    "                summary, step = sess.run([merged_summary_op, train_step],  feed_dict={X: batch_xs, Y_: batch_ys, lr:learning_rate, pkeep:0.75})\n",
    "                train_writer.add_summary(summary, i)\n",
    "                \n",
    "    print sess.run(accuracy, feed_dict={X: mnist.test.images, Y_: mnist.test.labels, pkeep:1.0})\n",
    "    train_writer.close()\n",
    "    test_writer.close()                             \n",
    "    sess.close()\n",
    "    \n",
    "nn_3_0()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
